package com.ustadzmenjawab.crawler;

import java.io.IOException;
import java.util.Set;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.google.gwt.dev.util.collect.HashSet;
import com.ustadzmenjawab.shared.Article;

public class LembagaFatwaMesirCrawler extends CrawlerRoot {
	public static final String HOME_URL = "http://www.dar-alifta.org/FatwaAnswer.aspx";
	public static final String PATTERN = "www.dar-alifta.org";
	public static final String[] SECTION_URLS = {
		"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=23&MuftiType=1&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=24&MuftiType=1&LangID=5"
		,"http://www.dar-alifta.org/ViewCategory.aspx?ID=14"
		,"http://www.dar-alifta.org/ViewCategory.aspx?ID=16"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=58&MuftiType=&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=59&MuftiType=1&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=60&MuftiType=&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=61&MuftiType=&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=62&MuftiType=1&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=63&MuftiType=1&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=64&MuftiType=1&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=87&MuftiType=1&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=88&MuftiType=1&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=89&MuftiType=1&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=90&MuftiType=1&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=65&MuftiType=1&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=66&MuftiType=1&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=67&MuftiType=1&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=69&MuftiType=1&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=70&MuftiType=1&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=71&MuftiType=1&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=72&MuftiType=1&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=73&MuftiType=1&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=74&MuftiType=1&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=75&MuftiType=1&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=76&MuftiType=1&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=77&MuftiType=1&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=91&MuftiType=1&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=92&MuftiType=1&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=93&MuftiType=1&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=94&MuftiType=1&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=18&MuftiType=1&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=35&MuftiType=1&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=36&MuftiType=1&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=37&MuftiType=1&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=39&MuftiType=1&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=41&MuftiType=1&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=44&MuftiType=1&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=51&MuftiType=1&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=52&MuftiType=1&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=53&MuftiType=1&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=54&MuftiType=1&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=55&MuftiType=1&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=78&MuftiType=1&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=21&MuftiType=1&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=30&MuftiType=1&LangID=5"
		,"http://www.dar-alifta.org/ViewCategory.aspx?ID=31&MuftiType=1&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=84&MuftiType=1&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=86&MuftiType=1&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=33&MuftiType=1&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=34&MuftiType=1&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=80&MuftiType=1&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=81&MuftiType=1&LangID=5"
		,"http://www.dar-alifta.org/ViewCategory.aspx?ID=82&MuftiType=1&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=83&MuftiType=1&LangID=5"
		,"http://www.dar-alifta.org/ViewCategory.aspx?ID=14&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=22&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=23&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=24&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategory.aspx?ID=16&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=32&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=57&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=58&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=59&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=60&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=61&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=62&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=63&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=64&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=87&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=88&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=89&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=90&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategory.aspx?ID=17&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=65&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=66&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=67&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=69&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=70&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=71&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=72&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=73&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=74&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=75&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=76&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=77&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=91&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=92&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=93&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=94&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=35&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=36&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=37&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=39&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=41&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategory.aspx?ID=20&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=44&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=51&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=52&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=53&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=54&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=55&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=78&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=21&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategory.aspx?ID=29&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=30&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategory.aspx?ID=31&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=84&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=86&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=33&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=34&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=80&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=81&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategory.aspx?ID=82&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=83&MuftiType=2&LangID=5"
		,"http://www.dar-alifta.org/ViewCategory.aspx?ID=14&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=22&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=23&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=24&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategory.aspx?ID=16&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=32&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=57&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=58&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=59&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=60&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=61&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=62&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=63&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=64&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=87&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=88&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=89&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=90&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategory.aspx?ID=17&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=65&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=66&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=67&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=69&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=70&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=71&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=72&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=73&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=74&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=75&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=76&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=77&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=91&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=92&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=93&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=94&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=35&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=36&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=37&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=39&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=41&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategory.aspx?ID=20&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=44&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=51&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=52&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=53&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=54&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=55&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=78&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=21&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategory.aspx?ID=29&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=30&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategory.aspx?ID=31&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=84&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=86&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=33&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=34&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=80&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=81&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategory.aspx?ID=82&MuftiType=3&LangID=5"
		,"http://www.dar-alifta.org/ViewCategoryFatawa.aspx?ID=83&MuftiType=3&LangID=5"
	};
	@Override
	public boolean isRelevantURL(String url) {
		if(url.contains("#"))
			return false;
		if((url.contains("/ViewFatwa.aspx") && url.contains("LangID=5")))
			return true;
		return false;
	}

	@Override
	public boolean isRelevantPage(String content) {
		if(getQuestion(content) != null){
			return true;
		}
		return false;
	}

	public boolean isRelevantPage(String url, String content){
		if((url.contains("/ViewFatwa.aspx") && url.contains("LangID=5"))){
			return isRelevantPage(content);
		}
		return false; 
	}

	@Override
	public String getTitle(String body) {
		Document doc = Jsoup.parse(body);
		Element p = doc.select("h2[class*=contentheading]").first();
		return p.text();
	}

	@Override
	public String getQuestion(String body) {
		Document doc = Jsoup.parse(body);
		Element p = doc.select("div[class*=QuestionClass]").first();
		return p.text();
	}

	@Override
	public String getArticle(String body) {
		Document doc = Jsoup.parse(body);
		Element p = doc.select("td[id*=tdAnswer]").first();
		return getQuestion(body) + " " + p.text();

	}

	@Override
	public boolean insertContent(String url, String content) {
		try{
			Article article = new Article();
			String title = getTitle(content);
			if(isTitleExist(title, "dar-alifta.org")){
				return false;
			}
			article.setJudul(title);
			article.setKategori("shariah");
			article.setSumber("dar-alifta.org");
			article.setTahun("2011");
			article.setUrl(url);
			article.setUstadz("Lembaga Fatwa Mesir");
			article.setArtikel(getArticle(content));
			article.setPertanyaan(getQuestion(content));
			db.insertArticle(article);
			return true;
		}catch(Exception e){
			System.out.println(" exception "+e.getClass() +" :" + e.getMessage());
			e.printStackTrace();
			insertFailure(url,"exception "+e.getClass() +" :" + e.getMessage());
			return false;
		}
	}

	public static void main(String[] args){
		LembagaFatwaMesirCrawler cr = new LembagaFatwaMesirCrawler();
		for(String categoryURL : SECTION_URLS){
			try{
				Document doc = Jsoup.connect(categoryURL).timeout(0).get();
				Elements unvisited = doc.select("a[href]");
				Set<String> visited = new HashSet<String>();
				while(unvisited.size() > 0){
					Element link = unvisited.first();
					try{
						String absHref = link.attr("abs:href");
						unvisited.remove(link);
						if(!visited.contains(absHref.toLowerCase())){
							visited.add(absHref.toLowerCase());
							if(cr.isRelevantURL(absHref)){
								Document page = Jsoup.connect(absHref).timeout(0).get();
								String content = page.html();
								Elements pageLinks = page.select("a[href]");
								for(Element newlink : pageLinks){
									String newLinkStr = newlink.attr("abs:href").toLowerCase();
									if(!visited.contains(newLinkStr) && cr.isRelevantURL(newLinkStr))
									{
										//unvisited.add(newlink);
									}
								}
								if(cr.isRelevantPage(absHref,content)){
									System.out.println(absHref + " is a relevant page");

									if(!cr.isUrlExist(absHref)){
										cr.insertContent(absHref, content);
									}
								}else{
									System.out.println(absHref + " is not a relevant page");
								}
							}
						}
					}catch(Exception e){
						System.out.println(" exception "+e.getClass() +" :" + e.getMessage());
						e.printStackTrace();
					}
				}
			}catch(Exception e){
				System.out.println(" exception "+e.getClass() +" :" + e.getMessage());
				e.printStackTrace();
			}
		}

	}
}
